import pandas as pd

df = pd.read_csv('heart_2020_cleaned.csv')
display(df)

pd.set_option('future.no_silent_downcasting', True)
binary_columns = [
    'HeartDisease', 'Smoking', 'AlcoholDrinking', 'Stroke',
    'DiffWalking', 'PhysicalActivity', 'Asthma',
    'KidneyDisease', 'SkinCancer'
]
# Map Yes/No to 1/0
for column in binary_columns:
    if column == 'KidneyDisease':
        df['KidneyDisease'] = df['KidneyDisease'].replace({'Yes': 1, 'No': 0}).astype(int)

    else:
        df[column] = df[column].map({'Yes': 1, 'No': 0})


def get_age_group_10_years(age_category):
    """
    Takes the original age category and returns a new 10-year category

    Parameters:
    age_category (str): Original age category (e.g. "18-24", "25-29" etc.)

    Returns:
    str: New age category ("18-29", "30-39" etc.)
    """
    if age_category in ['18-24', '25-29']:
        return '18-29'
    elif age_category in ['30-34', '35-39']:
        return '30-39'
    elif age_category in ['40-44', '45-49']:
        return '40-49'
    elif age_category in ['50-54', '55-59']:
        return '50-59'
    elif age_category in ['60-64', '65-69']:
        return '60-69'
    elif age_category in ['70-74', '75-79']:
        return '70-79'
    elif age_category == '80 or older':
        return '80+'
    else:
        return 'Unknown'


# Add new age group column
df['AgeCategory'] = df['AgeCategory'].apply(get_age_group_10_years)

display(df)

df = df[(df['SleepTime'] >= 1) & (df['SleepTime'] <= 16)]

import matplotlib.pyplot as plt
import seaborn as sns

# Create the figure
plt.figure(figsize=(13, 6))

# Create histogram/count plot with updated syntax
sns.countplot(data=df, x='HeartDisease', hue='HeartDisease', palette='Set2', legend=False)

# Calculate percentages
total = len(df['HeartDisease'])
percentages = df['HeartDisease'].value_counts(normalize=True) * 100

# Add percentage labels on top of each bar
for i, percentage in enumerate(percentages):
    plt.text(i, df['HeartDisease'].value_counts()[i],
             f'{percentage:.1f}%',
             horizontalalignment='center',
             verticalalignment='bottom')

# Customize the plot
plt.title('Distribution of Heart Disease Cases', pad=20)
plt.xlabel('Heart Disease')
plt.ylabel('Count')

# Show plot
plt.show()

sns.countplot(data=df, x='Race', hue='HeartDisease', palette='YlOrBr')
plt.xlabel('Race')
plt.ylabel('Frequency')
# Rotate x-axis labels for better readability
plt.xticks(rotation=45, ha='right')
plt.show()

# Remove Race column from dataset as it doesn't provide significant predictive value
df = df.drop('Race', axis=1)
print(f"Race column removed. Dataset now has {df.shape[1]} columns.")
display(df)

Race column removed. Dataset now has 17 columns.

sns.countplot(data=df, x='Sex', hue='HeartDisease', palette='YlOrBr')
plt.xlabel('Sex')
plt.ylabel('Frequency')
plt.show()

# create ordered list of age categories
age_order = ['18-29', '30-39', '40-49', '50-59', '60-69', '70-79', '80+']

# create the countplot with ordered categories
sns.countplot(data=df, x='AgeCategory', hue='HeartDisease',
              palette='YlOrBr', order=age_order)
plt.xlabel('Age Category')
plt.ylabel('Frequency')
plt.title('Heart Disease Distribution by Age Category')
plt.xticks(rotation=45)  # Rotate labels for better readability
plt.show()

df_copy = df.copy()

# Sex mapping
df_copy['Sex'] = df_copy['Sex'].replace({'Male': 1, 'Female': 0})

# Age mapping
age_map = {
    '18-29': 1,
    '30-39': 2,
    '40-49': 3,
    '50-59': 4,
    '60-69': 5,
    '70-79': 6,
    '80+': 7
}
df_copy['AgeCategory'] = df_copy['AgeCategory'].replace(age_map)

# Diabetic mapping
diabetic_mapping = {
    'No': 0,
    'No, borderline diabetes': 1,
    'Yes (during pregnancy)': 2,
    'Yes': 3
}
df_copy['Diabetic'] = df_copy['Diabetic'].replace(diabetic_mapping)

# General Health mapping
genhealth_mapping = {
    'Poor': 0,
    'Fair': 1,
    'Good': 2,
    'Very good': 3,
    'Excellent': 4
}

df_copy['GenHealth'] = df_copy['GenHealth'].replace(genhealth_mapping)
df_copy['Sex'] = df_copy['Sex'].replace({'Male': 1, 'Female': 0}).astype(int)
display(df_copy)

# Create correlation matrix
correlation_matrix = df_copy.corr()

# Create a heatmap of the correlation matrix
plt.figure(figsize=(10, 8))
sns.heatmap(correlation_matrix,
            annot=True,  # show correlation values
            cmap='coolwarm',  # color scheme
            center=0,  # center the colormap at 0
            fmt='.2f',  # show 2 decimal places
            square=True,  # make the plot square-shaped
            vmin=-1, vmax=1)  # set the range of values

plt.title('Correlation Matrix of Variables')
plt.tight_layout()
plt.show()

from statsmodels.stats.outliers_influence import variance_inflation_factor
from statsmodels.tools.tools import add_constant
import numpy as np

# First, let's create a new DataFrame with just the columns we need
# This helps us avoid modifying the original DataFrame
df_vif = pd.DataFrame()

# Convert GenHealth to numeric values
genhealth_mapping = {
    'Poor': 0,
    'Fair': 1,
    'Good': 2,
    'Very good': 3,
    'Excellent': 4
}
df_vif['GenHealth_numeric'] = df['GenHealth'].replace(genhealth_mapping).astype(float)

# Ensure PhysicalHealth is numeric and handle any non-numeric values
df_vif['PhysicalHealth'] = pd.to_numeric(df['PhysicalHealth'], errors='coerce')

# Drop any rows with NaN values to ensure clean data for VIF calculation
df_vif = df_vif.dropna()

# Add a constant column for VIF calculation
X = add_constant(df_vif)

# Calculate VIF
vif_data = pd.DataFrame()
vif_data["Feature"] = X.columns
vif_data["VIF"] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]

# Print the VIF data
print("VIF Data:")
print(vif_data)

# If you want to calculate VIF after dropping a feature
X_dropped = add_constant(df_vif[['PhysicalHealth']])
vif_data_dropped = pd.DataFrame()
vif_data_dropped["Feature"] = X_dropped.columns
vif_data_dropped["VIF"] = [variance_inflation_factor(X_dropped.values, i) for i in range(X_dropped.shape[1])]

print("\nVIF Data after dropping GenHealth_numeric:")
print(vif_data_dropped)

VIF Data:
             Feature        VIF
0              const  10.639505
1  GenHealth_numeric   1.302905
2     PhysicalHealth   1.302905

VIF Data after dropping GenHealth_numeric:
          Feature       VIF
0           const  1.179549
1  PhysicalHealth  1.000000

from scipy.stats import ttest_ind

t_stat, p_value_ttest = ttest_ind(df['PhysicalActivity'], df['DiffWalking'])

print(f"T-statistic: {t_stat:.3f}")
print(f"P-value of T-test: {p_value_ttest:.3f}")

T-statistic: 664.727
P-value of T-test: 0.000

# read the data again to avoid changes that affect the next graph
df_smoke_drink = pd.read_csv('heart_2020_cleaned.csv')

# create a new column combining smoking and alcohol status
df_smoke_drink['Habits'] = (df_smoke_drink['Smoking'].map({'Yes': 'Smoker', 'No': 'Non-Smoker'}) + ', ' +
                            df_smoke_drink['AlcoholDrinking'].map({'Yes': 'Drinker', 'No': 'Non-Drinker'}))

# calculate heart disease percentage for each combination
heart_disease_stats = df_smoke_drink.groupby('Habits')['HeartDisease'].apply(
    lambda x: (x == 'Yes').mean() * 100).reset_index()

# sort values for better visualization
heart_disease_stats = heart_disease_stats.sort_values('HeartDisease')

# create a bar plot
plt.figure(figsize=(12, 6))
ax = sns.barplot(x='Habits',
                 y='HeartDisease',
                 data=heart_disease_stats,
                 color='skyblue')

# add percentage labels on top of each bar
for i, v in enumerate(heart_disease_stats['HeartDisease']):
    ax.text(i, v, f'{v:.1f}%', ha='center', va='bottom')

plt.title('Percentage of Heart Disease by Smoking and Alcohol Habits')
plt.xlabel('Lifestyle Habits')

plt.ylabel('Percentage of People with Heart Disease (%)')

plt.tight_layout()
plt.show()

# read the data again to avoid changes that affect the next graph
df_physic_gen = pd.read_csv('heart_2020_cleaned.csv')

# create a new column combining physical activity and general health status
df_physic_gen['Health_Status'] = (df_physic_gen['PhysicalActivity'].map({'Yes': 'Active', 'No': 'Inactive'}) + ', ' +
                                  df_physic_gen['GenHealth'])

# calculate heart disease percentage for each combination
heart_disease_stats = df_physic_gen.groupby('Health_Status')['HeartDisease'].apply(
    lambda x: (x == 'Yes').mean() * 100).reset_index()

# sort values for better visualization
heart_disease_stats = heart_disease_stats.sort_values('HeartDisease')

# create a bar plot
plt.figure(figsize=(12, 6))
ax = sns.barplot(x='Health_Status',
                 y='HeartDisease',
                 data=heart_disease_stats,
                 color='skyblue')

# add percentage labels on top of each bar
for i, v in enumerate(heart_disease_stats['HeartDisease']):
    ax.text(i, v, f'{v:.1f}%', ha='center', va='bottom')

plt.title('Percentage of Heart Disease by Physical Activity and General Health Status')
plt.xlabel('Health Status')
plt.ylabel('Percentage of People with Heart Disease (%)')

# Rotate x-axis labels for better readability
plt.xticks(rotation=45, ha='right')

plt.tight_layout()
plt.show()

# read the data again to avoid changes that affect the next graph
df_physic_mental = pd.read_csv('heart_2020_cleaned.csv')


# create categories for Physical and Mental Health
def categorize_health(value):
    if value == 0:
        return 'Perfect (0 days)'
    elif value <= 5:
        return '1-5 days'
    elif value <= 15:
        return '6-15 days'
    else:
        return 'Over 15 days'


# create new columns with categorized health values
df_physic_mental['PhysicalHealth_Cat'] = df_physic_mental['PhysicalHealth'].apply(categorize_health)
df_physic_mental['MentalHealth_Cat'] = df_physic_mental['MentalHealth'].apply(categorize_health)

# create a new column combining both health categories
df_physic_mental['Health_Status'] = 'Physical: ' + df_physic_mental['PhysicalHealth_Cat'] + ', Mental: ' + \
                                    df_physic_mental['MentalHealth_Cat']

# calculate heart disease percentage for each combination
heart_disease_stats = df_physic_mental.groupby('Health_Status')['HeartDisease'].apply(
    lambda x: (x == 'Yes').mean() * 100).reset_index()

# sort values for better visualization
heart_disease_stats = heart_disease_stats.sort_values('HeartDisease')

# create a bar plot
plt.figure(figsize=(15, 8))
ax = sns.barplot(x='Health_Status',
                 y='HeartDisease',
                 data=heart_disease_stats,
                 color='skyblue')

# add percentage labels on top of each bar
for i, v in enumerate(heart_disease_stats['HeartDisease']):
    ax.text(i, v, f'{v:.1f}%', ha='center', va='bottom')

plt.title('Percentage of Heart Disease by Physical and Mental Health Status')
plt.xlabel('Health Status')
plt.ylabel('Percentage of People with Heart Disease (%)')
plt.xticks(rotation=45, ha='right')

plt.tight_layout()
plt.show()

# read the data again to avoid changes that affect the next graph
df_diabetic_stroke = pd.read_csv('heart_2020_cleaned.csv')

# create a combination of health conditions
df_diabetic_stroke['Health_Conditions'] = (
        df_diabetic_stroke['Diabetic'].map({'Yes': 'Diabetic', 'No': 'Non-Diabetic'}) + ', ' +
        df_diabetic_stroke['Stroke'].map({'Yes': 'Stroke', 'No': 'No Stroke'}))

# calculate heart disease percentage for each combination
heart_disease_stats = df_diabetic_stroke.groupby('Health_Conditions')['HeartDisease'].apply(
    lambda x: (x == 'Yes').mean() * 100).reset_index()

# sort values for better visualization
heart_disease_stats = heart_disease_stats.sort_values('HeartDisease')

# create a bar plot
plt.figure(figsize=(12, 6))
ax = sns.barplot(x='Health_Conditions',
                 y='HeartDisease',
                 data=heart_disease_stats,
                 color='skyblue')

# add percentage labels on top of each bar
for i, v in enumerate(heart_disease_stats['HeartDisease']):
    ax.text(i, v, f'{v:.1f}%', ha='center', va='bottom')

plt.title('Percentage of Heart Disease by Diabetes and Stroke Status')
plt.xlabel('Health Conditions')
plt.ylabel('Percentage of People with Heart Disease (%)')

plt.tight_layout()
plt.show()

# create the density plot
plt.figure(figsize=(10, 6))
sns.kdeplot(data=df, x='BMI', hue='HeartDisease', fill=True, common_norm=False)

plt.title('BMI Distribution by Heart Disease Status')
plt.xlabel('BMI')
plt.ylabel('Density')

plt.tight_layout()
plt.show()

from scipy.stats import f_oneway

df_balanced = pd.read_csv('heart_2020_cleaned.csv')
df_balanced = df_balanced.dropna(subset=["BMI"])

df_filtered = df_balanced.dropna(subset=["BMI"])  # Remove NaNs from BMI
df_filtered["HeartDisease"] = df_filtered["HeartDisease"].map({"No": 0, "Yes": 1})
bmi_no_heart_disease = df_filtered[df_filtered["HeartDisease"] == 0]["BMI"]
bmi_with_heart_disease = df_filtered[df_filtered["HeartDisease"] == 1]["BMI"]


# Perform ANOVA test
stat, p_value = f_oneway(bmi_no_heart_disease, bmi_with_heart_disease)

# Print results
print(f"ANOVA Statistic: {stat:.4f}, p-value: {p_value:.4f}")

ANOVA Statistic: 860.4963, p-value: 0.0000

sns.kdeplot(data=df, x='KidneyDisease', hue='HeartDisease', fill=True, common_norm=False)

plt.title('KidneyDisease Distribution by Heart Disease Status')
plt.xlabel('KidneyDisease')
plt.ylabel('Density')

plt.tight_layout()
plt.show()

sns.kdeplot(data=df, x='SleepTime', hue='HeartDisease', fill=True, common_norm=False)

plt.title('Sleep Time Distribution by Heart Disease Status')
plt.xlabel('Hours of Sleep')
plt.ylabel('Density')

# Add vertical line for recommended sleep (8 hours)
plt.axvline(x=8, color='red', linestyle='--', alpha=0.5, label='Recommended Sleep (8h)')
plt.legend(title='Heart Disease')

plt.tight_layout()
plt.show()

# separate plots by Heart Disease status for BMI vs Sleep Time
plt.figure(figsize=(15, 6))
for i, condition in enumerate(df['HeartDisease'].unique()):
    plt.subplot(1, 2, i + 1)
    subset = df[df['HeartDisease'] == condition]
    sns.kdeplot(
        data=subset, x="BMI", y="SleepTime",
        fill=True, thresh=0, levels=20, cmap="rocket"
    )
    plt.title(f'BMI vs Sleep Time\nHeart Disease: {condition}')

plt.tight_layout()
plt.show()

df["SleepTime"].dropna(inplace=True)
# Group Sleep Time by Heart Disease Status
group1 = df[df["HeartDisease"] == 0]["SleepTime"]
group2 = df[df["HeartDisease"] == 1]["SleepTime"]

# Compute Mean and Standard Deviation for Each Group
mean1, mean2 = np.mean(group1), np.mean(group2)
std1, std2 = np.std(group1, ddof=1), np.std(group2, ddof=1)

# Compute Pooled Standard Deviation
n1, n2 = len(group1), len(group2)
sp = np.sqrt(((n1 - 1) * std1 ** 2 + (n2 - 1) * std2 ** 2) / (n1 + n2 - 2))

# Compute Cohen’s d
cohen_d = (mean1 - mean2) / sp
print(f"Cohen's d: {cohen_d:.4f} (Effect Size)")

Cohen's d: -0.0202 (Effect Size)

df = df.drop('SleepTime', axis=1)
print(f"SleepTime column removed. Dataset now has {df.shape[1]} columns.")

SleepTime column removed. Dataset now has 16 columns.

# create separate plots for each Heart Disease category
plt.figure(figsize=(15, 10))
for i, health in enumerate(df['HeartDisease'].unique()):
    plt.subplot(2, 3, i + 1)
    subset = df[df['HeartDisease'] == health]
    sns.kdeplot(
        data=subset, x="BMI", y="PhysicalHealth",
        fill=True, thresh=0, levels=15, cmap="viridis"
    )
    plt.title(f'BMI vs Physical Health\nHeart Disease: {health}')

plt.tight_layout()
plt.show()

# Physical and Mental Health Impact
plt.figure(figsize=(15, 6))
plt.subplot(1, 2, 1)
sns.kdeplot(
    data=df[df['HeartDisease'] == 1],
    x="PhysicalHealth", y="MentalHealth",
    fill=True, thresh=0, levels=20, cmap="rocket",
)
plt.title(f'Physical Health vs Mental Health\nHeart Disease: 1')

plt.subplot(1, 2, 2)
sns.kdeplot(
    data=df[df['HeartDisease'] == 0],
    x="PhysicalHealth", y="MentalHealth",
    fill=True, thresh=0, levels=20, cmap="rocket",
)
plt.title(f'Physical Health vs Mental Health\nHeart Disease: 0')
plt.tight_layout()
plt.show()

from statsmodels.stats.outliers_influence import variance_inflation_factor

# Copy the dataset
df_vif = df.copy()

# Select only numerical features for VIF analysis
selected_features = ["PhysicalHealth", "MentalHealth"]

# Create a new DataFrame with only selected features
df_vif = df_vif[selected_features]

# Drop any missing values
df_vif = df_vif.dropna()

# Compute VIF for each feature
vif_data = pd.DataFrame()
vif_data["Feature"] = df_vif.columns
vif_data["VIF"] = [variance_inflation_factor(df_vif.values, i) for i in range(len(df_vif.columns))]

# Display the results
print(vif_data)

          Feature       VIF
0  PhysicalHealth  1.200932
1    MentalHealth  1.200932

import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler

# First, reload the dataset
df_visualization = pd.read_csv('heart_2020_cleaned.csv')

plt.close('all')

# Convert HeartDisease column to consistent Yes/No strings
df_visualization['HeartDisease'] = df_visualization['HeartDisease'].map({'Yes': 'Yes', 'No': 'No'})

# Sample the data since plotting all points would be overwhelming
sample_size = 5000  # Adjust sample size as needed
df_sampled = df_visualization.sample(n=sample_size, random_state=42)

# Normalize features to 0-1 range for better visualization
scaler = MinMaxScaler()
scaled_features = scaler.fit_transform(df_sampled[['PhysicalHealth', 'MentalHealth', 'BMI']])
df_sampled[['PhysicalHealth_scaled', 'MentalHealth_scaled', 'BMI_scaled']] = scaled_features

# Create 3D plot with lighter background for better contrast
plt.close('all')
fig = plt.figure(figsize=(12, 10), facecolor='white')
ax = fig.add_subplot(111, projection='3d')

# Define distinct colors with high contrast
colors = {'No': 'blue', 'Yes': 'red'}

# Plot each category separately
for disease_status in df_sampled['HeartDisease'].unique():
    mask = df_sampled['HeartDisease'] == disease_status
    ax.scatter(
        df_sampled[mask]['PhysicalHealth_scaled'],
        df_sampled[mask]['MentalHealth_scaled'],
        df_sampled[mask]['BMI_scaled'],
        c=colors[disease_status],
        label=f'Heart Disease: {disease_status}',
        alpha=1.0,  # Full opacity for better visibility
        s=50  # Larger point size for visibility
    )

# Customize axes labels
ax.set_xlabel('Physical Health (normalized)', fontweight='bold')
ax.set_ylabel('Mental Health (normalized)', fontweight='bold')
ax.set_zlabel('BMI (normalized)', fontweight='bold')

plt.title('3D Relationship Between Physical Health, Mental Health, and BMI', fontsize=14)
plt.legend()

# Try different viewing angles to better see the data distribution
for angle in [(30, 45), (30, 120), (20, 210), (45, 300)]:
    ax.view_init(elev=angle[0], azim=angle[1])
    plt.draw()
    plt.pause(0.5)  # Display each angle for half a second

plt.show()

<Figure size 640x480 with 0 Axes>

<Figure size 640x480 with 0 Axes>

<Figure size 640x480 with 0 Axes>

plt.figure(figsize=(15, 8))
sns.boxplot(x="GenHealth", y="BMI", hue="HeartDisease", data=df)
plt.title('BMI Distribution by General Health and Heart Disease Status')
plt.show()

sns.set_theme(style="ticks", palette="pastel")
plt.figure(figsize=(15, 8))
sns.violinplot(x="Asthma", y="PhysicalHealth", hue="HeartDisease", data=df)
plt.title('Distribution of Physical Health Issues across Asthma and Heart Disease Status')
plt.xlabel('Asthma Status')
plt.ylabel('Physical Health Issues (days)')
plt.show()

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Read the dataset
df_age_astma = pd.read_csv('heart_2020_cleaned.csv')

# age mapping (converting categorical age to numeric)
age_mapping = {
    '18-24': 21,
    '25-29': 27,
    '30-34': 32,
    '35-39': 37,
    '40-44': 42,
    '45-49': 47,
    '50-54': 52,
    '55-59': 57,
    '60-64': 62,
    '65-69': 67,
    '70-74': 72,
    '75-79': 77,
    '80 or older': 82
}

# create new column with numeric age
df_age_astma['AgeNumeric'] = df_age_astma['AgeCategory'].map(age_mapping)

# filter data for skin cancer cases
astma = df_age_astma[df_age_astma['Asthma'] == 'Yes']

# create the joint plot with modified parameters
g = sns.jointplot(data=astma,
                  x="AgeNumeric",
                  y="BMI",
                  hue="HeartDisease",
                  kind="kde",
                  height=10)

plt.suptitle('Relationship between Age and BMI for Asthma Patients by Heart Disease Status',
             y=1.02,
             fontsize=12)

# improve axis labels
g.ax_joint.set_xlabel('Age', fontsize=10)
g.ax_joint.set_ylabel('BMI', fontsize=10)

plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

# Copy dataset
df_copy = df.copy()

# Convert categorical features to numerical
df_copy['Sex'] = df_copy['Sex'].replace({'Male': 1, 'Female': 0}).astype(int)

# Convert GenHealth to numerical values
genhealth_mapping = {
    'Poor': 0,
    'Fair': 1,
    'Good': 2,
    'Very good': 3,
    'Excellent': 4
}
df_copy['GenHealth'] = df_copy['GenHealth'].replace(genhealth_mapping)

# Select only the intended numerical columns
numerical_cols = ['BMI', 'PhysicalHealth', 'MentalHealth', 'GenHealth']
X = df_copy[numerical_cols]  # Ensure we are only using 4 features

# Standardize the features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply PCA
pca = PCA(n_components=2)
principal_components = pca.fit_transform(X_scaled)

# Create DataFrame for PCA results
principalDf = pd.DataFrame(data=principal_components, columns=['PC1', 'PC2'])

# Add HeartDisease and Sex columns for analysis
principalDf['HeartDisease'] = df_copy['HeartDisease']
principalDf['Sex'] = df_copy['Sex']

# Plot PCA scatterplot
plt.figure(figsize=(15, 10))
sns.scatterplot(data=principalDf,
                x="PC1",
                y="PC2",
                hue="Sex",
                palette="Set1",
                alpha=0.7)

plt.title('PCA of Health Metrics Colored by Sex')
plt.show()

# create a DataFrame with principal components
principalDf2 = pd.DataFrame(data=principal_components, columns=['PC 1', 'PC 2'])

# Add AgeCategory column for coloring (using the existing 10-year grouping)
principalDf2['AgeCategory'] = df_copy['AgeCategory']

# Create the scatter plot
plt.figure(figsize=(15, 10))
sns.scatterplot(data=principalDf2,
                x="PC 1",
                y="PC 2",
                hue="AgeCategory",
                palette="Set1",
                alpha=0.7)

# Customize the plot
plt.title('PCA of Health Metrics Colored by Age Category (10-year groups)', fontsize=14)
plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=12)
plt.tight_layout()

# Display explained variance
explained_variance = pca.explained_variance_ratio_
print(f"Explained variance ratio: {explained_variance}")
print(f"Total explained variance: {sum(explained_variance):.2f}")

plt.show()

Explained variance ratio: [0.44070492 0.24075338]
Total explained variance: 0.68

from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
from imblearn.over_sampling import SMOTE
import pandas as pd
from sklearn.preprocessing import LabelEncoder

# Suppress warnings
warnings.filterwarnings("ignore")

# Simple mapping for all categorical columns
for col in df.columns:
    if df[col].dtype == 'object':
        # Explicit mapping for binary columns
        if set(df[col].unique()).issubset({'Yes', 'No'}):
            df[col] = df[col].map({'Yes': 1, 'No': 0})
        else:
            # LabelEncoder for other categorical columns
            le = LabelEncoder()
            df[col] = le.fit_transform(df[col])

# Making sure that we deleted those columns
df = df.drop(['Race', 'SleepTime'], axis=1, errors='ignore')

# Prepare features and target
X = df.drop('HeartDisease', axis=1)
y = df['HeartDisease']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Apply SMOTE to balance the training dataset
smote = SMOTE(random_state=42)
X_train_balanced, y_train_balanced = smote.fit_resample(X_train, y_train)

print(f"Original training set distribution: {pd.Series(y_train).value_counts()}")
print(f"Balanced training set distribution: {pd.Series(y_train_balanced).value_counts()}")

# Visualize the class distribution before and after balancing
plt.figure(figsize=(12, 5))

# Before balancing
plt.subplot(1, 2, 1)
sns.countplot(x=y_train, palette='Set2')
plt.title('Heart Disease Distribution Before Balancing')
plt.xlabel('Heart Disease')
plt.ylabel('Count')

# After balancing
plt.subplot(1, 2, 2)
sns.countplot(x=y_train_balanced, palette='Set2')
plt.title('Heart Disease Distribution After SMOTE')
plt.xlabel('Heart Disease')
plt.ylabel('Count')
plt.tight_layout()
plt.show()

Original training set distribution: HeartDisease
0    233802
1     21846
Name: count, dtype: int64
Balanced training set distribution: HeartDisease
0    233802
1    233802
Name: count, dtype: int64

# Scale numeric features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train_balanced)
X_test_scaled = scaler.transform(X_test)

print("Data is now ready for model training:")
print(f"X_train_scaled shape: {X_train_scaled.shape}")
print(f"y_train_balanced shape: {y_train_balanced.shape}")
print(f"X_test_scaled shape: {X_test_scaled.shape}")
print(f"y_test shape: {y_test.shape}")

Data is now ready for model training:
X_train_scaled shape: (467604, 15)
y_train_balanced shape: (467604,)
X_test_scaled shape: (63913, 15)
y_test shape: (63913,)

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, precision_score, recall_score, f1_score
import matplotlib.pyplot as plt
import numpy as np

# Train on original data
model_orig = LogisticRegression(max_iter=1000)
model_orig.fit(X_train, y_train)
y_pred_orig = model_orig.predict(X_test)

# Train on balanced data
model_balanced = LogisticRegression(max_iter=1000)
model_balanced.fit(X_train_balanced, y_train_balanced)
y_pred_balanced = model_balanced.predict(X_test)

# Compare results
print("Logistic Regression Results:")
print("\nBefore SMOTE:")
print(classification_report(y_test, y_pred_orig))

print("\nAfter SMOTE:")
print(classification_report(y_test, y_pred_balanced))

# Create simple bar chart comparison for heart disease detection (class 1)
metrics = ['Precision', 'Recall', 'F1-Score']
before_values = [
    precision_score(y_test, y_pred_orig, pos_label=1),
    recall_score(y_test, y_pred_orig, pos_label=1),
    f1_score(y_test, y_pred_orig, pos_label=1)
]
after_values = [
    precision_score(y_test, y_pred_balanced, pos_label=1),
    recall_score(y_test, y_pred_balanced, pos_label=1),
    f1_score(y_test, y_pred_balanced, pos_label=1)
]

# Simple bar chart
plt.figure(figsize=(8, 5))
x = np.arange(len(metrics))
width = 0.35

plt.bar(x - width / 2, before_values, width, label='Before SMOTE')
plt.bar(x + width / 2, after_values, width, label='After SMOTE')

plt.ylabel('Score')
plt.title('Heart Disease Detection Performance')
plt.xticks(x, metrics)
plt.legend()
plt.ylim(0, 1)

plt.show()

Logistic Regression Results:

Before SMOTE:
              precision    recall  f1-score   support

           0       0.92      0.99      0.96     58435
           1       0.52      0.09      0.16      5478

    accuracy                           0.91     63913
   macro avg       0.72      0.54      0.56     63913
weighted avg       0.89      0.91      0.89     63913


After SMOTE:
              precision    recall  f1-score   support

           0       0.97      0.70      0.81     58435
           1       0.19      0.73      0.30      5478

    accuracy                           0.70     63913
   macro avg       0.58      0.72      0.56     63913
weighted avg       0.90      0.70      0.77     63913

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, precision_score, recall_score, f1_score
import matplotlib.pyplot as plt
import numpy as np

# Train on original data
rf_orig = RandomForestClassifier(n_estimators=100, random_state=42)
rf_orig.fit(X_train, y_train)
y_pred_orig = rf_orig.predict(X_test)

# Train on balanced data
rf_balanced = RandomForestClassifier(n_estimators=100, random_state=42)
rf_balanced.fit(X_train_balanced, y_train_balanced)
y_pred_balanced = rf_balanced.predict(X_test)

# Compare results
print("Random Forest Results:")
print("\nBefore SMOTE:")
print(classification_report(y_test, y_pred_orig))

print("\nAfter SMOTE:")
print(classification_report(y_test, y_pred_balanced))

# Create simple bar chart comparison for heart disease detection (class 1)
metrics = ['Precision', 'Recall', 'F1-Score']
before_values = [
    precision_score(y_test, y_pred_orig, pos_label=1),
    recall_score(y_test, y_pred_orig, pos_label=1),
    f1_score(y_test, y_pred_orig, pos_label=1)
]
after_values = [
    precision_score(y_test, y_pred_balanced, pos_label=1),
    recall_score(y_test, y_pred_balanced, pos_label=1),
    f1_score(y_test, y_pred_balanced, pos_label=1)
]

# Simple bar chart
plt.figure(figsize=(8, 5))
x = np.arange(len(metrics))
width = 0.35

plt.bar(x - width / 2, before_values, width, label='Before SMOTE')
plt.bar(x + width / 2, after_values, width, label='After SMOTE')

plt.ylabel('Score')
plt.title('Heart Disease Detection Performance')
plt.xticks(x, metrics)
plt.legend()
plt.ylim(0, 1)

plt.show()

Random Forest Results:

Before SMOTE:
              precision    recall  f1-score   support

           0       0.92      0.97      0.95     58435
           1       0.32      0.15      0.20      5478

    accuracy                           0.90     63913
   macro avg       0.62      0.56      0.57     63913
weighted avg       0.87      0.90      0.88     63913


After SMOTE:
              precision    recall  f1-score   support

           0       0.93      0.87      0.90     58435
           1       0.20      0.35      0.26      5478

    accuracy                           0.82     63913
   macro avg       0.57      0.61      0.58     63913
weighted avg       0.87      0.82      0.85     63913

from matplotlib.patches import Circle, FancyArrowPatch

def plot_neural_network(layers_sizes, title="Neural Network Architecture"):
    # Set figure size
    fig = plt.figure(figsize=(12, 9))
    ax = fig.gca()
    ax.axis('off')

    # Layer positions
    n_layers = len(layers_sizes)
    layer_centers = np.linspace(0.2, 0.8, n_layers)

    # Vertical spacing for neurons
    max_neurons = max(layers_sizes)
    v_spacing = 0.9 / max(max_neurons, 1)

    # Draw layers and neurons
    for i, layer_size in enumerate(layers_sizes):
        layer_name = "Input" if i == 0 else "Output" if i == n_layers - 1 else f"Hidden {i}"

        # Neurons in current layer
        for j in range(layer_size):
            # Limit the number of displayed neurons if there are too many
            if layer_size <= 10 or (j < 4 or j >= layer_size - 4):
                x = layer_centers[i]
                y = 0.5 + (j - layer_size / 2) * v_spacing

                # Draw neuron
                circle = Circle((x, y), 0.02,
                             color='#3498db' if i == 0 else '#e74c3c' if i == n_layers - 1 else '#2ecc71',
                             ec='k', zorder=4)
                ax.add_patch(circle)

                # Connect to next layer
                if i < n_layers - 1:
                    next_layer_size = layers_sizes[i + 1]
                    next_x = layer_centers[i + 1]

                    # Connect to all neurons in next layer
                    for k in range(next_layer_size):
                        if next_layer_size <= 10 or (k < 4 or k >= next_layer_size - 4):
                            next_y = 0.5 + (k - next_layer_size / 2) * v_spacing
                            line = FancyArrowPatch((x, y), (next_x, next_y),
                                                  connectionstyle="arc3,rad=0.1",
                                                  arrowstyle="-", color='gray', alpha=0.2)
                            ax.add_patch(line)

            # Add dots to indicate more neurons not shown
            if layer_size > 10 and j == 4:
                plt.text(layer_centers[i], 0.5, "...", fontsize=20, ha='center')

        # Layer labels
        plt.text(layer_centers[i], 0.15, f"{layer_name}\n({layer_size} neurons)",
                ha='center', va='top', fontsize=12,
                bbox=dict(boxstyle='round', facecolor='white', alpha=0.5))

    plt.title(title, fontsize=14)
    plt.tight_layout()
    plt.show()

# Draw our neural network
input_size = 16  # Number of features in your model (adjust as needed)
plot_neural_network([input_size, 100, 50, 1], "MLP Architecture for Heart Disease Prediction")

from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report, precision_score, recall_score, f1_score
import matplotlib.pyplot as plt
import numpy as np

# Neural Network before SMOTE
mlp_orig = MLPClassifier(
    hidden_layer_sizes=(100, 50),  # Two hidden layers with 100 and 50 neurons
    activation='relu',             # ReLU activation function
    solver='adam',                 # Optimization algorithm
    alpha=0.0001,                  # Regularization parameter
    batch_size='auto',             # Automatic batch size
    learning_rate='adaptive',      # Adaptive learning rate
    max_iter=500,                  # Maximum number of iterations
    random_state=42                # Random seed for reproducibility
)

# Train the model on imbalanced data
mlp_orig.fit(X_train, y_train)
y_pred_orig = mlp_orig.predict(X_test)

# Neural Network after SMOTE
mlp_balanced = MLPClassifier(
    hidden_layer_sizes=(100, 50),
    activation='relu',
    solver='adam',
    alpha=0.0001,
    batch_size='auto',
    learning_rate='adaptive',
    max_iter=500,
    random_state=42
)

# Train the model on SMOTE-balanced data
mlp_balanced.fit(X_train_balanced, y_train_balanced)
y_pred_balanced = mlp_balanced.predict(X_test)

# Compare results
print("Neural Network Results:")
print("\nBefore SMOTE:")
print(classification_report(y_test, y_pred_orig))

print("\nAfter SMOTE:")
print(classification_report(y_test, y_pred_balanced))

# Create a bar chart for comparing heart disease detection (class 1)
metrics = ['Precision', 'Recall', 'F1-Score']
before_values = [
    precision_score(y_test, y_pred_orig, pos_label=1),
    recall_score(y_test, y_pred_orig, pos_label=1),
    f1_score(y_test, y_pred_orig, pos_label=1)
]
after_values = [
    precision_score(y_test, y_pred_balanced, pos_label=1),
    recall_score(y_test, y_pred_balanced, pos_label=1),
    f1_score(y_test, y_pred_balanced, pos_label=1)
]

# Create simple bar chart
plt.figure(figsize=(10, 6))
x = np.arange(len(metrics))
width = 0.35

plt.bar(x - width / 2, before_values, width, label='Before SMOTE', color='skyblue')
plt.bar(x + width / 2, after_values, width, label='After SMOTE', color='salmon')

# Add numerical values above bars
for i, v in enumerate(before_values):
    plt.text(i - width/2, v + 0.02, f'{v:.2f}', ha='center', va='bottom')

for i, v in enumerate(after_values):
    plt.text(i + width/2, v + 0.02, f'{v:.2f}', ha='center', va='bottom')

plt.ylabel('Score')
plt.title('Neural Network Performance on Heart Disease Detection')
plt.xticks(x, metrics)
plt.legend()
plt.ylim(0, 1)
plt.grid(axis='y', linestyle='--', alpha=0.7)

plt.tight_layout()
plt.show()

Neural Network Results:

Before SMOTE:
              precision    recall  f1-score   support

           0       0.92      0.99      0.96     58435
           1       0.53      0.11      0.18      5478

    accuracy                           0.92     63913
   macro avg       0.73      0.55      0.57     63913
weighted avg       0.89      0.92      0.89     63913


After SMOTE:
              precision    recall  f1-score   support

           0       0.96      0.75      0.84     58435
           1       0.20      0.65      0.30      5478

    accuracy                           0.74     63913
   macro avg       0.58      0.70      0.57     63913
weighted avg       0.89      0.74      0.80     63913

from xgboost import XGBClassifier
from sklearn.metrics import classification_report, precision_score, recall_score, f1_score
import matplotlib.pyplot as plt
import numpy as np
# Train on original data
xgb_orig = XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.1, random_state=42)
xgb_orig.fit(X_train, y_train)
y_pred_orig = xgb_orig.predict(X_test)

# Train on balanced data
xgb_balanced = XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.1, random_state=42)
xgb_balanced.fit(X_train_balanced, y_train_balanced)
y_pred_balanced = xgb_balanced.predict(X_test)

# Compare results
print("XGBoost Results:")
print("\nBefore SMOTE:")
print(classification_report(y_test, y_pred_orig))

print("\nAfter SMOTE:")
print(classification_report(y_test, y_pred_balanced))

# Create simple bar chart comparison for heart disease detection (class 1)
metrics = ['Precision', 'Recall', 'F1-Score']
before_values = [
    precision_score(y_test, y_pred_orig, pos_label=1),
    recall_score(y_test, y_pred_orig, pos_label=1),
    f1_score(y_test, y_pred_orig, pos_label=1)
]
after_values = [
    precision_score(y_test, y_pred_balanced, pos_label=1),
    recall_score(y_test, y_pred_balanced, pos_label=1),
    f1_score(y_test, y_pred_balanced, pos_label=1)
]

# Simple bar chart
plt.figure(figsize=(8, 5))
x = np.arange(len(metrics))
width = 0.35

plt.bar(x - width / 2, before_values, width, label='Before SMOTE')
plt.bar(x + width / 2, after_values, width, label='After SMOTE')

plt.ylabel('Score')
plt.title('Heart Disease Detection Performance')
plt.xticks(x, metrics)
plt.legend()
plt.ylim(0, 1)

plt.show()

XGBoost Results:

Before SMOTE:
              precision    recall  f1-score   support

           0       0.92      0.99      0.96     58435
           1       0.57      0.08      0.15      5478

    accuracy                           0.92     63913
   macro avg       0.75      0.54      0.55     63913
weighted avg       0.89      0.92      0.89     63913


After SMOTE:
              precision    recall  f1-score   support

           0       0.97      0.73      0.83     58435
           1       0.20      0.73      0.32      5478

    accuracy                           0.73     63913
   macro avg       0.58      0.73      0.58     63913
weighted avg       0.90      0.73      0.79     63913

import pandas as pd
from sklearn.cluster import AgglomerativeClustering
from scipy.cluster.hierarchy import dendrogram, linkage
import matplotlib.pyplot as plt
from sklearn.metrics.pairwise import pairwise_distances
from scipy.spatial.distance import squareform
import numpy as np

pd.set_option('future.no_silent_downcasting', True)

# Check if X_train is a DataFrame or numpy array
if isinstance(X_train, pd.DataFrame):
    # If DataFrame, use .iloc for indexing
    np.random.seed(42)
    sample_indices = np.random.choice(len(X_train), size=5000, replace=False)
    X_train_sample = X_train.iloc[sample_indices].values
    y_train_sample = y_train.iloc[sample_indices].values if isinstance(y_train, pd.Series) else y_train[sample_indices]
else:
    # If numpy array, use direct indexing
    np.random.seed(42)
    sample_indices = np.random.choice(len(X_train), size=5000, replace=False)
    X_train_sample = X_train[sample_indices]
    y_train_sample = y_train[sample_indices]

# Calculate distance matrix (using euclidean distance for symmetric matrix)
distances_orig = pairwise_distances(X_train_sample, metric='euclidean')

# Make sure the matrix is symmetric
distances_orig = (distances_orig + distances_orig.T) / 2

# Perform hierarchical clustering with 5 clusters
n_clusters = 5
cluster_orig = AgglomerativeClustering(n_clusters=n_clusters,
                                 metric='precomputed',
                                 linkage='complete')
clusters_orig = cluster_orig.fit_predict(distances_orig)

# Calculate heart disease rates by cluster
df_results_orig = pd.DataFrame({
    'Cluster': clusters_orig,
    'HeartDisease': y_train_sample
})
heart_disease_by_cluster_orig = df_results_orig.groupby('Cluster')['HeartDisease'].mean() * 100

# ---- Hierarchical Clustering on SMOTE Balanced Data ----
# Sample the balanced dataset
if isinstance(X_train_balanced, pd.DataFrame):
    # If DataFrame, use .iloc for indexing
    np.random.seed(42)
    balanced_sample_indices = np.random.choice(len(X_train_balanced), size=5000, replace=False)
    X_train_balanced_sample = X_train_balanced.iloc[balanced_sample_indices].values
    y_train_balanced_sample = y_train_balanced.iloc[balanced_sample_indices].values if isinstance(y_train_balanced, pd.Series) else y_train_balanced[balanced_sample_indices]
else:
    # If numpy array, use direct indexing
    np.random.seed(42)
    balanced_sample_indices = np.random.choice(len(X_train_balanced), size=5000, replace=False)
    X_train_balanced_sample = X_train_balanced[balanced_sample_indices]
    y_train_balanced_sample = y_train_balanced[balanced_sample_indices]

# Calculate distance matrix (using euclidean distance for symmetric matrix)
distances_balanced = pairwise_distances(X_train_balanced_sample, metric='euclidean')

# Make sure the matrix is symmetric
distances_balanced = (distances_balanced + distances_balanced.T) / 2

# Perform hierarchical clustering with 5 clusters
cluster_balanced = AgglomerativeClustering(n_clusters=n_clusters,
                                    metric='precomputed',
                                    linkage='complete')
clusters_balanced = cluster_balanced.fit_predict(distances_balanced)

# Calculate heart disease rates by cluster
df_results_bal = pd.DataFrame({
    'Cluster': clusters_balanced,
    'HeartDisease': y_train_balanced_sample
})
heart_disease_by_cluster_bal = df_results_bal.groupby('Cluster')['HeartDisease'].mean() * 100

# ---- Create dendrograms -----
# Prepare data for dendrograms
condensed_dist_orig = squareform(distances_orig)
linkage_matrix_orig = linkage(condensed_dist_orig, method='complete')

condensed_dist_bal = squareform(distances_balanced)
linkage_matrix_bal = linkage(condensed_dist_bal, method='complete')

# Create dendrograms side by side
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 7))

# Dendrogram before SMOTE (left)
dendrogram(linkage_matrix_orig, ax=ax1)
ax1.set_title('Hierarchical Clustering Dendrogram (Before SMOTE)')
ax1.set_xlabel('Sample Index')
ax1.set_ylabel('Distance')

# Dendrogram after SMOTE (right)
dendrogram(linkage_matrix_bal, ax=ax2)
ax2.set_title('Hierarchical Clustering Dendrogram (After SMOTE)')
ax2.set_xlabel('Sample Index')
ax2.set_ylabel('Distance')

plt.tight_layout()
plt.show()

# ---- Create heart disease rate graphs ----
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 6))

# Before SMOTE - left side
bars_orig = ax1.bar(range(n_clusters), heart_disease_by_cluster_orig, color='skyblue')
ax1.set_title('Heart Disease Rate by Cluster (Before SMOTE)')
ax1.set_xlabel('Cluster')
ax1.set_ylabel('Heart Disease Rate (%)')
ax1.set_xticks(range(n_clusters))

# Add percentage labels on bars
for bar, percentage in zip(bars_orig, heart_disease_by_cluster_orig):
    ax1.text(bar.get_x() + bar.get_width()/2.,
            bar.get_height() + 0.5,
            f'{percentage:.1f}%',
            ha='center', va='bottom')

# After SMOTE - right side
bars_bal = ax2.bar(range(n_clusters), heart_disease_by_cluster_bal, color='skyblue')
ax2.set_title('Heart Disease Rate by Cluster (After SMOTE)')
ax2.set_xlabel('Cluster')
ax2.set_ylabel('Heart Disease Rate (%)')
ax2.set_xticks(range(n_clusters))

# Add percentage labels on bars
for bar, percentage in zip(bars_bal, heart_disease_by_cluster_bal):
    ax2.text(bar.get_x() + bar.get_width()/2.,
            bar.get_height() + 0.5,
            f'{percentage:.1f}%',
            ha='center', va='bottom')

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Create DataFrame with model performance metrics after SMOTE
metrics_data = {
    'Model': ['Logistic Regression', 'Random Forest', 'Neural Network', 'XGBoost'],
    'Accuracy': [0.70, 0.82, 0.74, 0.73],
    'Precision (No HD)': [0.97, 0.93, 0.96, 0.97],
    'Recall (No HD)': [0.70, 0.87, 0.75, 0.73],
    'F1 (No HD)': [0.81, 0.90, 0.84, 0.83],
    'Precision (HD)': [0.19, 0.20, 0.20, 0.20],
    'Recall (HD)': [0.73, 0.35, 0.65, 0.73],
    'F1 (HD)': [0.30, 0.26, 0.30, 0.32]
}

metrics_df = pd.DataFrame(metrics_data)

# Display the comparison table
print("Model Performance Comparison (After SMOTE):")
print(metrics_df.to_string(index=False))

# Create a bar plot for comparison
plt.figure(figsize=(15, 8))
metrics_to_plot = ['Accuracy', 'Precision (HD)', 'Recall (HD)', 'F1 (HD)']
plot_data = metrics_df.melt(id_vars=['Model'],
                            value_vars=metrics_to_plot,
                            var_name='Metric',
                            value_name='Score')

# Plot the comparison using seaborn
sns.barplot(x='Model', y='Score', hue='Metric', data=plot_data)
plt.title('Model Performance Comparison (After SMOTE)')
plt.xticks(rotation=45)
plt.ylim(0, 1)
plt.legend(title='Metric', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.tight_layout()
plt.show()

Model Performance Comparison (After SMOTE):
              Model  Accuracy  Precision (No HD)  Recall (No HD)  F1 (No HD)  Precision (HD)  Recall (HD)  F1 (HD)
Logistic Regression      0.70               0.97            0.70        0.81            0.19         0.73     0.30
      Random Forest      0.82               0.93            0.87        0.90            0.20         0.35     0.26
     Neural Network      0.74               0.96            0.75        0.84            0.20         0.65     0.30
            XGBoost      0.73               0.97            0.73        0.83            0.20         0.73     0.32

import shap
import os
import warnings
import sys

# Disable all warnings
warnings.filterwarnings('ignore')

# Redirect stdout temporarily to suppress progress output
original_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')

# Disable kmeans iterations display
shap.utils._legacy.kmeans.kmeans = lambda X, k, **kwargs: (X[:k], None)

try:
    # Convert X_test back to DataFrame with column names
    X_test_df = pd.DataFrame(X_test, columns=X.columns)

    # Create SHAP explainer without verbose parameter
    explainer = shap.Explainer(xgb_balanced, X_test_df)
    shap_values = explainer(X_test_df)
finally:
    # Restore stdout
    sys.stdout.close()
    sys.stdout = original_stdout

# Create beeswarm plot
shap.plots.beeswarm(shap_values)

100%|===================| 63774/63913 [03:24<00:00]

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import shap
import warnings
import matplotlib as mpl

# Suppress warnings
warnings.filterwarnings("ignore")

# Predictions using existing balanced model
y_pred = xgb_balanced.predict(X_test)
y_pred_proba = xgb_balanced.predict_proba(X_test)[:, 1]

# Create results DataFrame
results_df = pd.DataFrame({
    'True_Label': y_test.values,
    'Predicted': y_pred,
    'Probability': y_pred_proba
})

# Find representative cases
true_positive_indices = results_df[(results_df['True_Label'] == 1) & (results_df['Predicted'] == 1)].index
true_positive = true_positive_indices[0] if len(true_positive_indices) > 0 else None

true_negative_indices = results_df[(results_df['True_Label'] == 0) & (results_df['Predicted'] == 0)].index
true_negative = true_negative_indices[0] if len(true_negative_indices) > 0 else None

false_positive_indices = results_df[(results_df['True_Label'] == 0) & (results_df['Predicted'] == 1)].index
false_positive = false_positive_indices[0] if len(false_positive_indices) > 0 else None

false_negative_indices = results_df[(results_df['True_Label'] == 1) & (results_df['Predicted'] == 0)].index
false_negative = false_negative_indices[0] if len(false_negative_indices) > 0 else None

# Collect case indices and labels
case_indices = [idx for idx in [true_positive, true_negative, false_positive, false_negative] if idx is not None]
case_labels = ['True Positive', 'True Negative', 'False Positive', 'False Negative'][:len(case_indices)]

# Create SHAP explainer
X_test_df = pd.DataFrame(X_test.reset_index(drop=True), columns=X.columns)
explainer = shap.TreeExplainer(xgb_balanced)

# Define consistent figure size for all plots - making wider to handle the left margin
fixed_figsize = (6.5, 3.5)

# Set consistent values for all plots
border_width = 2
border_color = 'black'
font_size = 10

# Reset any previous matplotlib settings
mpl.rcParams.update(mpl.rcParamsDefault)
# Force all plots to have exactly the same dimensions
mpl.rcParams['savefig.bbox'] = 'standard'  # Don't use tight bbox
mpl.rcParams['savefig.pad_inches'] = 0.1   # Consistent padding

# Create and save waterfall plots with consistent borders
for i, idx in enumerate(case_indices):
    # Get feature values for this specific case
    x_instance = X_test_df.iloc[[idx]]

    # Get SHAP values
    shap_values = explainer(x_instance)

    # Create figure with fixed size
    fig = plt.figure(figsize=fixed_figsize)

    # Create waterfall plot with smaller inner plot
    shap.plots.waterfall(
        shap_values[0],
        max_display=5,
        show=False
    )

    # Add title
    case_type = case_labels[i]
    true_label = "Has Heart Disease" if results_df['True_Label'].iloc[idx] == 1 else "No Heart Disease"
    pred_label = "Has Heart Disease" if results_df['Predicted'].iloc[idx] == 1 else "No Heart Disease"
    prob = results_df['Probability'].iloc[idx] * 100

    plt.title(f'Case Analysis: {case_type}\n'
              f'True Label: {true_label}, Prediction: {pred_label}, Probability: {prob:.1f}%',
              fontsize=font_size, fontweight='bold')

    # Add a border to the figure
    fig.patch.set_edgecolor(border_color)
    fig.patch.set_linewidth(border_width)

    plt.subplots_adjust(left=0.25, right=0.80, top=0.80, bottom=0.20)

    # Save with consistent size
    filename = f"case_{case_type.replace(' ', '_')}.png"
    plt.savefig(filename, dpi=100, bbox_inches=None)  # No auto-cropping
    plt.close()

# Display all saved charts
from IPython.display import Image, display
for case_type in case_labels:
    display(Image(filename=f"case_{case_type.replace(' ', '_')}.png"))

	HeartDisease	BMI	Smoking	AlcoholDrinking	Stroke	PhysicalHealth	MentalHealth	DiffWalking	Sex	AgeCategory	Race	Diabetic	PhysicalActivity	GenHealth	SleepTime	Asthma	KidneyDisease	SkinCancer
0	No	16.60	Yes	No	No	3.0	30.0	No	Female	55-59	White	Yes	Yes	Very good	5.0	Yes	No	Yes
1	No	20.34	No	No	Yes	0.0	0.0	No	Female	80 or older	White	No	Yes	Very good	7.0	No	No	No
2	No	26.58	Yes	No	No	20.0	30.0	No	Male	65-69	White	Yes	Yes	Fair	8.0	Yes	No	No
3	No	24.21	No	No	No	0.0	0.0	No	Female	75-79	White	No	No	Good	6.0	No	No	Yes
4	No	23.71	No	No	No	28.0	0.0	Yes	Female	40-44	White	No	Yes	Very good	8.0	No	No	No
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
319790	Yes	27.41	Yes	No	No	7.0	0.0	Yes	Male	60-64	Hispanic	Yes	No	Fair	6.0	Yes	No	No
319791	No	29.84	Yes	No	No	0.0	0.0	No	Male	35-39	Hispanic	No	Yes	Very good	5.0	Yes	No	No
319792	No	24.24	No	No	No	0.0	0.0	No	Female	45-49	Hispanic	No	Yes	Good	6.0	No	No	No
319793	No	32.81	No	No	No	0.0	0.0	No	Female	25-29	Hispanic	No	No	Good	12.0	No	No	No
319794	No	46.56	No	No	No	0.0	0.0	No	Female	80 or older	Hispanic	No	Yes	Good	8.0	No	No	No

Intro¶

The Problem¶

The Importance of the Solution¶

How We're Going to Do It¶

Data Collection and Selection Process¶

Data Analysis¶

Data Processing Overview¶

Data Preprocessing:¶

Converting Categorical Variables¶

Data Cleaning: Filter Sleep Duration Values¶

Summary of Data Processing¶

Exploratory Data Analysis (EDA) Overview¶

EDA¶

Basic histogram of the prediction column¶

Distribution of heart disease cases in our population¶

Distribution of Heart Disease by Race¶

Distribution of Heart Disease by Sex¶

Distribution of Heart Disease by Age Category¶

Correlation Matrix of Health Variables¶

Investigating Multicollinearity Using VIF Analysis¶

Statistical Significance Testing: Comparing Physical Activity and Difficulty Walking¶

Heart Disease Distribution by Lifestyle Habits¶

Distribution of Heart Disease by Physical Activity and Health Status¶

Distribution of Heart Disease by Physical and Mental Health Days¶

Heart Disease by Diabetes and Stroke Status¶

BMI Distribution by Heart Disease Status¶

ANOVA Testing: Comparing BMI Distribution Between Heart Disease Groups¶

Kidney Disease Distribution by Heart Disease Status¶

Sleep Time Distribution by Heart Disease Status¶

BMI vs Sleep Time by Heart Disease Status¶

Cohen's d Effect Size Analysis: Sleep Time and Heart Disease¶

BMI vs Physical Health by Heart Disease Status¶

Physical Health vs Mental Health by Heart Disease Status¶

Multicollinearity Analysis: Variance Inflation Factor (VIF)¶

3D Relationship between Physical Health, Mental Health, BMI, and Heart Disease¶

BMI Distribution by General Health and Heart Disease Status¶

Physical Health Distribution by Asthma and Heart Disease Status¶

Relationship between Age and BMI for Asthma Patients by Heart Disease Status¶

PCA (Principal Component Analysis)¶

Focused PCA with Health Metrics Colored by Sex Category¶

Focused PCA with Health Metrics Colored by Age Category¶

Summary of Exploratory Data Analysis¶

Model Building and Training Overview¶

Data Balancing using SMOTE¶

Arrange the data to input and target and train and set¶

Apply SMOTE to Training Data¶

Scale Features¶

Model Building and Algorithm Selection¶

Logistic Regression¶

Random Forest¶

Neural Network¶

XGBoost¶

Hierarchical Clustering¶

Comparing the Models¶

Model Summary and Analysis¶

Interpretability of the Models¶

Enhancing SHAP Analysis: Case-Specific Investigation¶

Analysis of SHAP Results for Specific Cases¶

Summary and Thoughts for the future and Final Taught¶